{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write fake accuracies\n",
    "import tensorflow as tf\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "\n",
    "accuracy = [0.1, 0.4, 0.6, 0.8, 0.9, 0.95] # fake accuracy values\n",
    "acc_var = tf.Variable(0, dtype=tf.float32) # variable that holds accuracy\n",
    "acc_summ = tf.summary.scalar('Accuracy', acc_var) # summary to write to TensorBoard\n",
    "\n",
    "sess = tf.Session()\n",
    "for step, acc in enumerate(accuracy):\n",
    "    sess.run(acc_var.assign(acc)) # update accuracy variable\n",
    "    \n",
    "    writer.add_summary(sess.run(acc_summ), step) # add summary\n",
    "\n",
    "writer.flush() # make sure everything is written to disk\n",
    "writer.close() # not really needed, but good habit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a basic graph\n",
    "import tensorflow as tf\n",
    "tf.reset_default_graph()\n",
    "\n",
    "a = tf.Variable(2, name='a')\n",
    "b = tf.Variable(3, name='b')\n",
    "c = tf.add(a, b, name='Add')\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs', graph=sess.graph)\n",
    "\n",
    "print(sess.run(c)) # -> 5\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a giant graph\n",
    "import tensorflow as tf\n",
    "tf.reset_default_graph()\n",
    "\n",
    "a = tf.Variable(2.0, name='a')\n",
    "b = tf.Variable(3.0, name='b')\n",
    "c = tf.Variable(7.0, name='c')\n",
    "\n",
    "d = tf.multiply(a, b, name='d-mul')\n",
    "e = tf.add(b, c, name='e-add')\n",
    "f = tf.subtract(e, a, name='f-sub')\n",
    "g = tf.multiply(d, b, name='g-mul')\n",
    "h = tf.divide(g, d, name='h-div')\n",
    "i = tf.add(h, f, name='i-add')\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs', graph=sess.graph)\n",
    "\n",
    "print(sess.run(i)) # -> 11\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a U-Net generator\n",
    "import tensorflow as tf\n",
    "tf.reset_default_graph()\n",
    "\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Activation, Concatenate, Conv2D, Conv2DTranspose, Input, LeakyReLU\n",
    "from keras.initializers import RandomNormal\n",
    "from keras_contrib.layers import InstanceNormalization # pip install git+https://www.github.com/keras-team/keras-contrib.git\n",
    "\n",
    "img_rows, img_cols, channels = 256, 256, 3\n",
    "weight_initializer = RandomNormal(stddev=0.02)\n",
    "\n",
    "# \"dk denotes a 3×3 Convolution-InstanceNorm-ReLU with k filters and stride 2\"\n",
    "def dk(k, use_instancenorm=True):\n",
    "    block = Sequential()\n",
    "    block.add(Conv2D(k, (3, 3), strides=2, padding='same', kernel_initializer=weight_initializer))\n",
    "    if use_instancenorm:\n",
    "        block.add(InstanceNormalization(axis=-1))\n",
    "    block.add(Activation('relu'))\n",
    "\n",
    "    return block\n",
    "\n",
    "# \"uk denotes a 3×3 fractional-strided-ConvolutionInstanceNorm-ReLU layer with k filters and stride ½\"\n",
    "def uk(k):\n",
    "    block = Sequential()\n",
    "    block.add(Conv2DTranspose(k, (3, 3), strides=2, padding='same', kernel_initializer=weight_initializer))\n",
    "    block.add(InstanceNormalization(axis=-1))\n",
    "    block.add(Activation('relu'))\n",
    "\n",
    "    return block\n",
    "\n",
    "def generator():\n",
    "    gen_input = Input(shape=(img_rows, img_cols, channels))\n",
    "    \n",
    "    # Layers for the encoder part of the model\n",
    "    encoder_layers = [\n",
    "        dk(64, False),\n",
    "        dk(128),\n",
    "        dk(256),\n",
    "        dk(512),\n",
    "        dk(512),\n",
    "        dk(512),\n",
    "        dk(512),\n",
    "        dk(512)\n",
    "    ]\n",
    "\n",
    "    # Layers for the decoder part of the model\n",
    "    decoder_layers = [\n",
    "        uk(512),\n",
    "        uk(512),\n",
    "        uk(512),\n",
    "        uk(512),\n",
    "        uk(256),\n",
    "        uk(128),\n",
    "        uk(64)\n",
    "    ]\n",
    "\n",
    "    gen = gen_input\n",
    "\n",
    "    # Add all the encoder layers, and keep track of them for skip connections\n",
    "    skips = []\n",
    "    for layer in encoder_layers:\n",
    "        gen = layer(gen)\n",
    "        skips.append(gen)\n",
    "    \n",
    "    skips = skips[::-1][1:] # Reverse for looping and get rid of the layer that directly connects to decoder\n",
    "\n",
    "    # Add all the decoder layers and skip connections\n",
    "    for skip_layer, layer in zip(skips, decoder_layers):\n",
    "        gen = layer(gen)\n",
    "        gen = Concatenate()([gen, skip_layer])\n",
    "\n",
    "    \n",
    "    # Final layer\n",
    "    gen = Conv2DTranspose(channels, (3, 3), strides=2, padding='same', kernel_initializer=weight_initializer, activation='tanh')(gen)\n",
    "    \n",
    "    # Compose model\n",
    "    return Model(gen_input, gen)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "model = generator()\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs', graph=sess.graph)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a simple normal distribution\n",
    "import tensorflow as tf\n",
    "\n",
    "normal_dist = tf.random_normal(shape=(10, 10, 10, 10, 10), mean=0, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "summary = tf.summary.histogram(\"NormalDistribution\", normal_dist) # summary that logs tensor\n",
    "\n",
    "sess = tf.Session()\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "\n",
    "writer.add_summary(sess.run(summary))\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stack 100 normal distributions\n",
    "import tensorflow as tf\n",
    "\n",
    "mean = tf.placeholder(tf.float32)\n",
    "\n",
    "normal_dist = tf.random_normal(shape=(10, 10, 10, 10, 10), mean=mean, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "summary = tf.summary.histogram(\"NormalDistribution\", normal_dist) # summary that logs tensor\n",
    "\n",
    "sess = tf.Session()\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "\n",
    "for i in range(100):\n",
    "    writer.add_summary(sess.run(summary, feed_dict={mean: i/50}), global_step=0)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize a multimodal distributon\n",
    "import tensorflow as tf\n",
    "\n",
    "mean = tf.placeholder(tf.float32)\n",
    "\n",
    "normal_dist_0 = tf.random_normal(shape=(10000,), mean=-mean, stddev=0.5)\n",
    "normal_dist_1 = tf.random_normal(shape=(10000,), mean=mean, stddev=0.5)\n",
    "\n",
    "normal_dist_cat = tf.concat([normal_dist_0, normal_dist_1], axis=0) # concatenate both normal dists\n",
    "summary = tf.summary.histogram(\"MultimodalDistribution\", normal_dist_cat) # summary that logs tensor\n",
    "\n",
    "sess = tf.Session()\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "\n",
    "for i in range(100):\n",
    "    writer.add_summary(sess.run(summary, feed_dict={mean: i/20}), global_step=0)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize some random nosie\n",
    "import tensorflow as tf\n",
    "\n",
    "image = tf.random_uniform((1, 210, 160, 3), 0, 256, dtype=tf.float32) # batch_size, height, width, channels\n",
    "img_summary = tf.summary.image(\"InputImage\", image)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "x = sess.run(img_summary)\n",
    "writer.add_summary(x)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visuaize the first 4 examples of MNIST\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "\n",
    "images = x_train[:4].reshape((-1, 28, 28, 1))\n",
    "img_summary = tf.summary.image(\"TrainingExamples\", images, max_outputs=4)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "x = sess.run(img_summary)\n",
    "writer.add_summary(x)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display some text\n",
    "import tensorflow as tf\n",
    "\n",
    "text = tf.convert_to_tensor('Hello world!')\n",
    "txt_summary = tf.summary.text(\"Text\", text)\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "x = sess.run(txt_summary)\n",
    "writer.add_summary(x)\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge summaries\n",
    "import tensorflow as tf\n",
    "\n",
    "# Image summary\n",
    "image = tf.random_uniform((1, 210, 160, 3), 0, 256, dtype=tf.float32) # batch_size, height, width, channels\n",
    "img_summary = tf.summary.image(\"InputImage\", image, max_outputs=10)\n",
    "\n",
    "# Histogram summary\n",
    "normal_dist = tf.random_normal(shape=(10, 10, 10, 10, 10), mean=0, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "hist_summary = tf.summary.histogram(\"NormalDistribution\", normal_dist) # summary that logs tensor\n",
    "\n",
    "# Merge summaries\n",
    "merged_summ = tf.summary.merge([img_summary, hist_summary])\n",
    "\n",
    "# Start session\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "# Init. writer and add summary\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "writer.add_summary(sess.run(merged_summ))\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# JUPYTER-NOTEBOOK NOTE: If you have not restarted your Notebook before running this, this will report ALL\n",
    "# summaries from previous cells included\n",
    "\n",
    "# Merge summaries\n",
    "import tensorflow as tf\n",
    "\n",
    "# Image summary\n",
    "image = tf.random_uniform((1, 210, 160, 3), 0, 256, dtype=tf.float32) # batch_size, height, width, channels\n",
    "img_summary = tf.summary.image(\"InputImage\", image, max_outputs=10)\n",
    "\n",
    "# Histogram summary\n",
    "normal_dist = tf.random_normal(shape=(10, 10, 10, 10, 10), mean=0, stddev=1) # tensor to be logged, shape is irrelevant\n",
    "hist_summary = tf.summary.histogram(\"NormalDistribution\", normal_dist) # summary that logs tensor\n",
    "\n",
    "# Merge summaries\n",
    "merged_summ = tf.summary.merge_all() # no need to specify which summaries to merge\n",
    "\n",
    "# Start session\n",
    "sess = tf.Session()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "# Init. writer and add summary\n",
    "writer = tf.summary.FileWriter('./graphs')\n",
    "writer.add_summary(sess.run(merged_summ))\n",
    "\n",
    "writer.flush()\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the TensorBoard callback in a Keras model\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, Flatten, Dense, MaxPooling2D, Dropout\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape((-1, 28, 28, 1))\n",
    "x_test = x_test.reshape((-1, 28, 28, 1))\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, 10)\n",
    "y_test = keras.utils.to_categorical(y_test, 10)\n",
    "\n",
    "model = Sequential([\n",
    "    Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
    "    Conv2D(64, (3, 3), activation='relu'),\n",
    "    MaxPooling2D((2, 2)),\n",
    "    Dropout(0.25),\n",
    "    Flatten(),\n",
    "    Dense(128), # we will visualize neurons in this layer\n",
    "    Dropout(0.5),\n",
    "    Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "tensorboard_callback = keras.callbacks.TensorBoard('./graphs',\n",
    "    histogram_freq=1, # how often to compute activation and weight histograms\n",
    "    write_graph=True, # visualize the graph\n",
    "    write_grads=True, # visual gradient histogram\n",
    "    write_images=True, # visualize model weights as an image\n",
    "    # embeddings_freq=1, # how often to visualize embeddings\n",
    "    # embeddings_layer_names=['...'], # names of embedding layers to visualize; wouldn't work on this model\n",
    "    update_freq='epoch' # update TensorBoard every epoch\n",
    ")\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=32,\n",
    "          epochs=15,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test),\n",
    "          callbacks=[tensorboard_callback])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
